#!/usr/bin/env ruby -wKU
#encoding: UTF-8

require 'nokogiri'
require 'open-uri'

require 'backend/model/TChapter'
require 'backend/model/TVolume'
require 'backend/model/TNovel'
require 'backend/utils/NumberUtils'

module Crawler

  $HOME_PAGE = "http://tieba.baidu.com"

  #${name} should be the tieba name
  $TIEBA_QUERY_TEMPLATE = "/f?ie=utf-8&kw=${name}"

  # ISSUE, tieba html structure change
  $THREAD_XPATH_ON_TOP = "//ul[@id='thread_top_list']//div[contains(@class, 'threadlist_title')]/a"
  $THREAD_XPATH_NOT_ON_TOP = "//ul[@id='thread_list']//div[contains(@class, 'threadlist_title')]/a"
  #$THREAD_XPATH_ON_TOP = "//ul[@id='thread_list']/li[contains(@class,'thread_top')]/*/*/*/div[contains(@class, 'threadlist_text')]/a"
  #$THREAD_XPATH_NOT_ON_TOP = "//ul[@id='thread_list']/li/*/*/*/div[contains(@class, 'threadlist_text')]/a"

  #THREAD_TITLE = "title"
  $THREAD_LINK = "href"

  $MAX_THREADS_TO_PROCESS = 5

  $VOLUME_CHAPTER_PATTERN = "(第(.*?)卷(.*?))?第(.*?)章(.*)"

  # FIX: <a></a> cannot be parsed if text() is present
  #$POST_CONTENT_XPATH = "//div[@id='j_p_postlist']/div[1]/*/*/*/div[@id[starts-with(., 'post_content')]]/text()"
  $POST_CONTENT_XPATH = "//div[@id='j_p_postlist']/div[1]/*/*/*/div[@id[starts-with(., 'post_content')]]"
  
  
  class HtmlParser

    # Use following section to avoid this warning:
    #Users/gujun/.rvm/rubies/ruby-2.0.0-p353/lib/ruby/site_ruby/2.0.0/rubygems/core_ext/kernel_require.rb:55: warning: loading in progress,
    #circular require considered harmful - /Users/gujun/.rvm/rubies/ruby-2.0.0-p353/lib/ruby/2.0.0/net/http.rb
    #from /Users/gujun/Ruby/tieba-novel-crawler/ruby/backend/crawler/TiebaCrawler.rb:35:in `block (2 levels) in crawl'
    #from /Users/gujun/Ruby/tieba-novel-crawler/ruby/backend/crawler/HtmlParser.rb:42:in `fetch'
    #from /Users/gujun/.rvm/rubies/ruby-2.0.0-p353/lib/ruby/2.0.0/open-uri.rb:34:in `open'
    #from /Users/gujun/.rvm/rubies/ruby-2.0.0-p353/lib/ruby/2.0.0/open-uri.rb:688:in `open'
    #from /Users/gujun/.rvm/rubies/ruby-2.0.0-p353/lib/ruby/2.0.0/open-uri.rb:149:in `open_uri'
    #from /Users/gujun/.rvm/rubies/ruby-2.0.0-p353/lib/ruby/2.0.0/open-uri.rb:208:in `open_loop'
    #from /Users/gujun/.rvm/rubies/ruby-2.0.0-p353/lib/ruby/2.0.0/open-uri.rb:208:in `catch'
    #from /Users/gujun/.rvm/rubies/ruby-2.0.0-p353/lib/ruby/2.0.0/open-uri.rb:210:in `block in open_loop'
    #from /Users/gujun/.rvm/rubies/ruby-2.0.0-p353/lib/ruby/2.0.0/open-uri.rb:708:in `buffer_open'
    #from /Users/gujun/.rvm/rubies/ruby-2.0.0-p353/lib/ruby/2.0.0/open-uri.rb:267:in `open_http'
    #from /Users/gujun/.rvm/rubies/ruby-2.0.0-p353/lib/ruby/site_ruby/2.0.0/rubygems/core_ext/kernel_require.rb:55:in `require'
    #from /Users/gujun/.rvm/rubies/ruby-2.0.0-p353/lib/ruby/site_ruby/2.0.0/rubygems/core_ext/kernel_require.rb:55:in `require'
    begin
      open($HOME_PAGE)
      #$logger.info(__FILE__) { __LINE__.to_s + " open #{$HOME_PAGE} once before fetch novel to load required module to avoid circular require which is considered harmful." }
    end

    def self.fetch(novel)
      $logger.info(__FILE__) { __LINE__.to_s + " fetch chapters for #{novel.state}" }

      chapters = []

      return chapters if novel.nil?

      novelUrl = tiebaQueryUrl(novel, true)
      return chapters if novelUrl.nil?

      # fetch novel page
	  begin
        novelPage = Nokogiri::HTML(open(novelUrl))
	  rescue Exception => e
		$logger.info(__FILE__) { __LINE__.to_s + " #{e.class} + failed for #{e.message}" }
		return chapters
	  end

      # find interesting items including volume and chapter info
      xpath = novel.onTop ? $THREAD_XPATH_ON_TOP : $THREAD_XPATH_NOT_ON_TOP
      chps = novelPage.xpath(xpath)

      # max number of interesting item to process
      num = chps.length > $MAX_THREADS_TO_PROCESS ? $MAX_THREADS_TO_PROCESS : chps.length
      chps = chps[0, num]
      
      # process each interesting item
      chps.each do |chp|
        
        # get interesting info
        if (match = chp.text.match(/#{$VOLUME_CHAPTER_PATTERN}/))
          volNoStr, volName, chpNoStr, chpName = match.captures[1, 4]
        end

        next if chpNoStr.nil? || chpName.nil?

        # init a volume instance
        # FIX issue on use max volume no
        volNo = volNoStr.nil? ? novel.maxVolNo : Utils::NumberUtils.cn_2_int(volNoStr.strip)
        # FIX issue on only use volNo to get volume since we may be unaware of volName yet
        # volume = novel.getVolume(volNo, volName)
        volume = novel.getVolumeByVolNo(volNo)
        if volume.nil?
          volume = Model::TVolume.new(novel)
          volume.volNo = Utils::NumberUtils.cn_2_int(volNoStr.strip) unless volNoStr.nil?
          volume.volName = volName.strip unless volName.nil?
        end
        
        chpNo = chpNoStr.nil? ? 0 : Utils::NumberUtils.cn_2_int(chpNoStr.strip)
        chapter = volume.getChapter(chpNo, chpName)
                
        # do nothing if this chapter exists
        next unless chapter.nil?

        # init a chapter instance
        chapter = Model::TChapter.new(volume)
        chapter.chpNo = chpNo # Utils::NumberUtils.cn_2_int(chpNoStr.strip)
        chapter.chpName = chpName.strip

        # get chapter link
        chpUrl = $HOME_PAGE + chp[$THREAD_LINK]
        
        # get chapter page
		begin
          chpPage = Nokogiri::HTML(open(chpUrl))
		rescue Exception => e
		  $logger.info(__FILE__) { __LINE__.to_s + " #{e.class} + failed for #{e.message}" }
		  next
		end

		# get title again in case it is too long that cannot be fully displayed in novel page
		chpTitle = chpPage.xpath('//*[@id="j_core_title_wrap"]/div/h1[@class="core_title_txt"]')
		if (match = chpTitle.text.match($VOLUME_CHAPTER_PATTERN))
          volNoStr, volName, chpNoStr, chpName = match.captures[1, 4]
          volume.volNo = Utils::NumberUtils.cn_2_int(volNoStr.strip) unless volNoStr.nil?
          volume.volName = volName.strip unless volName.nil?
		  
		  next if chpNoStr.nil? || chpName.nil?

		  chapter.chpNo = Utils::NumberUtils.cn_2_int(chpNoStr.strip)
		  chapter.chpName = chpName.strip
		end
        
        # find chapter content
        chpCnt = chpPage.xpath("//div[@id='j_p_postlist']/div[1]/*/*/*/div[@id[starts-with(., 'post_content')]]")

        # get chapter content
        content = ""
        chpCnt.children.each do |c|
          if c.text.strip == ""
            content << "\n"
          else
            content << c.text
          end
        end
        
        if content.length < 1000
          begin
            content = getContentRedirect('qidian', content)
          rescue Exception => e
		    $logger.info(__FILE__) { __LINE__.to_s + " #{e.class} + failed for #{e.message}" }
            next
          end
        end
        
        chapter.content = content

        chapters << chapter unless chapters.include? chapter

      end

      chapters

    end
    
    # def self.doTest
      # getContentRedirect('qidian', "short url: http://t.cn/8kdVt7p")
    # end

    private

    def self.tiebaQueryUrl(novel, encode)
      return nil if novel.nil? || novel.tiebaName.nil?
      replacement = encode ? URI.encode_www_form_component(novel.tiebaName) : novel.tiebaName
      $HOME_PAGE + $TIEBA_QUERY_TEMPLATE.gsub(/\${name}/, replacement)
    end
    
    def self.getContentRedirect(type, rawContent)
      # TODO: currently only support qidian.com
      content = ""
      if type == 'qidian'
        # try to find if there is some short url or mobile url to qidian.com
        # short url: http://t.cn/8kdVt7p
        # mobile url: http://m.qidian.com/book/bookreader.aspx?bookid=3022294&chapterid=50263779
        if (match = rawContent.match(/http:\/\/t\.cn\/\w+/))
		  begin
            chpPage = Nokogiri::HTML(open(match[0]))
		  rescue Exception => e
		    $logger.info(__FILE__) { __LINE__.to_s + " #{e.class} + failed for #{e.message}" }
			return content
		  end
          chpScript = chpPage.xpath("//*[@id='content']/script")[0]
          chpCntUri = chpScript.xpath("@src").text
          chpEncoding = chpScript.xpath("@charset").text
          
          # FIX: charset problem
          # refer to following network resources
          # http://ruby-china.org/topics/2484
          # http://stackoverflow.com/questions/1207335/how-to-convert-a-nethttp-response-to-a-certain-encoding-in-ruby-1-9-1
		  begin
            open(chpCntUri) do |file|
              chpCnt = file.gets.force_encoding(chpEncoding).encode("UTF-8", {:invalid => :replace, :undef => :replace, :replace => '?'})
              content = chpCnt.sub(/document.write\('/, "").gsub(/<p>/, "\n").sub(/'\);/, "")
            end
		  rescue Exception => e
		    $logger.info(__FILE__) { __LINE__.to_s + " #{e.class} + failed for #{e.message}" }
			return content
		  end
                    
          # this also works
          # require 'net/http'
          # response = Net::HTTP.get_response(URI.parse(chpCntUri))
          # chpCnt = response.body.force_encoding(chpEncoding).encode("UTF-8", {:invalid => :replace, :undef => :replace, :replace => '?'})
          # content = chpCnt.sub(/document.write\('/, "").gsub(/<p>/, "\n").sub(/'\);/, "")
          # puts content
          
          # Use nokogiri will miss some text if met text that cannot be converted
          # chpCnt = Nokogiri::HTML(open(chpCntUri), nil, chpEncoding)
          # chpCnt = Nokogiri::HTML(open(chpCntUri))
          # puts chpCnt.text.encode("UTF-8", {:invalid => :replace, :undef => :replace, :replace => '?'})
          # content = chpCnt.text.sub(/document.write\('/, "").gsub(/<p>/, "\n").sub(/'\);/, "")
          # puts content
        elsif (match = rawContent.match(/http:\/\/m\.qidian\.com\/book\/ReadChapter\.aspx\?bookid=\d+&chapterid=\d+/))
          # TODO: this seems do not work
          # puts match.captures
          # puts 'hello'
          # chpPage = Nokogiri::HTML(open("http://m.qidian.com/book/ReadChapter.aspx?bookid=3022294&chapterid=50263779"))
          # chpDiv = chpPage.xpath("//div[@class='chapter']") unless chpPage.nil?
          # puts chpDiv.text
          # chpDiv.text
        end
      else
        raise ArgumentError.new "Only support qidian currently."
      end
      content
    end

  end

  if __FILE__ == $0
    require 'logger'
    $logger = Logger.new('test_crawler.log')

    novel = Model::TNovel.new
    novel.tiebaName = "忘语"
    novel.novelName = "魔天记"
    novel.onTop = true
    # HtmlParser.fetch(novel)
    # HtmlParser.doTest
  end

end
